{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from __future__ import division\n",
    "import os\n",
    "import glob\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "86326\n"
     ]
    }
   ],
   "source": [
    "train_context = 'data/train.context'\n",
    "\n",
    "context_len = []\n",
    "\n",
    "with open(train_context) as fp:  \n",
    "    lines = fp.readlines()\n",
    "    print(len(lines))\n",
    "    for line in lines:\n",
    "        words = line.split()\n",
    "        context_len.append(words)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "86326\n",
      "[['sociologist', 'james', 'a.', 'beckford', ',', 'in', 'his', '1975', 'study', 'of', 'jehovah', \"'s\", 'witnesses', ',', 'classified', 'the', 'religion', \"'s\", 'organizational', 'structure', 'as', 'totalizing', ',', 'characterized', 'by', 'an', 'assertive', 'leadership', ',', 'specific', 'and', 'narrow', 'objectives', ',', 'control', 'over', 'competing', 'demands', 'on', 'members', \"'\", 'time', 'and', 'energy', ',', 'and', 'control', 'over', 'the', 'quality', 'of', 'new', 'members', '.', 'other', 'characteristics', 'of', 'the', 'classification', 'include', 'likelihood', 'of', 'friction', 'with', 'secular', 'authorities', ',', 'reluctance', 'to', 'co-operate', 'with', 'other', 'religious', 'organizations', ',', 'a', 'high', 'rate', 'of', 'membership', 'turnover', ',', 'a', 'low', 'rate', 'of', 'doctrinal', 'change', ',', 'and', 'strict', 'uniformity', 'of', 'beliefs', 'among', 'members', '.', 'beckford', 'identified', 'the', 'religion', \"'s\", 'chief', 'characteristics', 'as', 'historicism', '(', 'identifying', 'historical', 'events', 'as', 'relating', 'to', 'the', 'outworking', 'of', 'god', \"'s\", 'purpose', ')', ',', 'absolutism', '(', 'conviction', 'that', 'jehovah', \"'s\", 'witness', 'leaders', 'dispense', 'absolute', 'truth', ')', ',', 'activism', '(', 'capacity', 'to', 'motivate', 'members', 'to', 'perform', 'missionary', 'tasks', ')', ',', 'rationalism', '(', 'conviction', 'that', 'witness', 'doctrines', 'have', 'a', 'rational', 'basis', 'devoid', 'of', 'mystery', ')', ',', 'authoritarianism', '(', 'rigid', 'presentation', 'of', 'regulations', 'without', 'the', 'opportunity', 'for', 'criticism', ')', 'and', 'world', 'indifference', '(', 'rejection', 'of', 'certain', 'secular', 'requirements', 'and', 'medical', 'treatments', ')', '.'], ['the', 'geologic', 'record', 'of', 'the', 'proterozoic', '(', '2,500', 'to', '541', 'million', 'years', 'ago', ')', 'is', 'more', 'complete', 'than', 'that', 'for', 'the', 'preceding', 'archean', '.', 'in', 'contrast', 'to', 'the', 'deep-water', 'deposits', 'of', 'the', 'archean', ',', 'the', 'proterozoic', 'features', 'many', 'strata', 'that', 'were', 'laid', 'down', 'in', 'extensive', 'shallow', 'epicontinental', 'seas', ';', 'furthermore', ',', 'many', 'of', 'these', 'rocks', 'are', 'less', 'metamorphosed', 'than', 'archean-age', 'ones', ',', 'and', 'plenty', 'are', 'unaltered', '.', 'study', 'of', 'these', 'rocks', 'show', 'that', 'the', 'eon', 'featured', 'massive', ',', 'rapid', 'continental', 'accretion', '(', 'unique', 'to', 'the', 'proterozoic', ')', ',', 'supercontinent', 'cycles', ',', 'and', 'wholly', 'modern', 'orogenic', 'activity', '.', 'roughly', '750', 'million', 'years', 'ago', ',', 'the', 'earliest-known', 'supercontinent', 'rodinia', ',', 'began', 'to', 'break', 'apart', '.', 'the', 'continents', 'later', 'recombined', 'to', 'form', 'pannotia', ',', '600\\xe2\\x80\\x93540', 'ma', '.'], ['ottoman', 'turkish', 'was', 'the', 'official', 'language', 'of', 'the', 'empire', '.', 'it', 'was', 'an', 'oghuz', 'turkic', 'language', 'highly', 'influenced', 'by', 'persian', 'and', 'arabic', '.', 'the', 'ottomans', 'had', 'several', 'influential', 'languages', ':', 'turkish', ',', 'spoken', 'by', 'the', 'majority', 'of', 'the', 'people', 'in', 'anatolia', 'and', 'by', 'the', 'majority', 'of', 'muslims', 'of', 'the', 'balkans', 'except', 'in', 'albania', 'and', 'bosnia', ';', 'persian', ',', 'only', 'spoken', 'by', 'the', 'educated', ';', 'arabic', ',', 'spoken', 'mainly', 'in', 'arabia', ',', 'north', 'africa', ',', 'iraq', ',', 'kuwait', ',', 'the', 'levant', 'and', 'parts', 'of', 'the', 'horn', 'of', 'africa', ';', 'and', 'somali', 'throughout', 'the', 'horn', 'of', 'africa', '.', 'in', 'the', 'last', 'two', 'centuries', ',', 'usage', 'of', 'these', 'became', 'limited', ',', 'though', ',', 'and', 'specific', ':', 'persian', 'served', 'mainly', 'as', 'a', 'literary', 'language', 'for', 'the', 'educated', ',', 'while', 'arabic', 'was', 'used', 'for', 'religious', 'rites', '.'], ['madonna', \"'s\", 'use', 'of', 'sexual', 'imagery', 'has', 'benefited', 'her', 'career', 'and', 'catalyzed', 'public', 'discourse', 'on', 'sexuality', 'and', 'feminism', '.', 'as', 'roger', 'chapman', 'documents', 'in', 'culture', 'wars', ':', 'an', 'encyclopedia', 'of', 'issues', ',', 'viewpoints', ',', 'and', 'voices', ',', 'volume', '1', '(', '2010', ')', ',', 'she', 'has', 'drawn', 'frequent', 'condemnation', 'from', 'religious', 'organizations', ',', 'social', 'conservatives', 'and', 'parental', 'watchdog', 'groups', 'for', 'her', 'use', 'of', 'explicit', ',', 'sexual', 'imagery', 'and', 'lyrics', ',', 'religious', 'symbolism', ',', 'and', 'otherwise', '\"', 'irreverent', '\"', 'behavior', 'in', 'her', 'live', 'performances', '.', 'the', 'times', 'wrote', 'that', 'she', 'had', '\"', 'started', 'a', 'revolution', 'amongst', 'women', 'in', 'music', '...', 'her', 'attitudes', 'and', 'opinions', 'on', 'sex', ',', 'nudity', ',', 'style', 'and', 'sexuality', 'forced', 'the', 'public', 'to', 'sit', 'up', 'and', 'take', 'notice', '.', '\"', 'professor', 'john', 'fiske', 'noted', 'that', 'the', 'sense', 'of', 'empowerment', 'that', 'madonna', 'offers', 'is', 'inextricably', 'connected', 'with', 'the', 'pleasure', 'of', 'exerting', 'some', 'control', 'over', 'the', 'meanings', 'of', 'self', ',', 'of', 'sexuality', ',', 'and', 'of', 'one', \"'s\", 'social', 'relations', '.', 'in', 'doing', 'gender', 'in', 'media', ',', 'art', 'and', 'culture', '(', '2009', ')', ',', 'the', 'authors', 'noted', 'that', 'madonna', ',', 'as', 'a', 'female', 'celebrity', ',', 'performer', ',', 'and', 'pop', 'icon', ',', 'is', 'able', 'to', 'unsettle', 'standing', 'feminist', 'reflections', 'and', 'debates', '.', 'according', 'to', 'lesbian', 'feminist', 'sheila', 'jeffreys', ',', 'madonna', 'represents', 'woman', \"'s\", 'occupancy', 'of', 'what', 'monique', 'wittig', 'calls', 'the', 'category', 'of', 'sex', ',', 'as', 'powerful', ',', 'and', 'appears', 'to', 'gleefully', 'embrace', 'the', 'performance', 'of', 'the', 'sexual', 'corv\\xc3\\xa9e', 'allotted', 'to', 'women', '.', 'professor', 'sut', 'jhally', 'has', 'referred', 'to', 'madonna', 'as', '\"', 'an', 'almost', 'sacred', 'feminist', 'icon', '.', '\"'], ['the', 'group', 'changed', 'their', 'name', 'to', 'destiny', \"'s\", 'child', 'in', '1996', ',', 'based', 'upon', 'a', 'passage', 'in', 'the', 'book', 'of', 'isaiah', '.', 'in', '1997', ',', 'destiny', \"'s\", 'child', 'released', 'their', 'major', 'label', 'debut', 'song', '\"', 'killing', 'time', '\"', 'on', 'the', 'soundtrack', 'to', 'the', '1997', 'film', ',', 'men', 'in', 'black', '.', 'the', 'following', 'year', ',', 'the', 'group', 'released', 'their', 'self-titled', 'debut', 'album', ',', 'scoring', 'their', 'first', 'major', 'hit', '\"', 'no', ',', 'no', ',', 'no', '\"', '.', 'the', 'album', 'established', 'the', 'group', 'as', 'a', 'viable', 'act', 'in', 'the', 'music', 'industry', ',', 'with', 'moderate', 'sales', 'and', 'winning', 'the', 'group', 'three', 'soul', 'train', 'lady', 'of', 'soul', 'awards', 'for', 'best', 'r', '&', 'b/soul', 'album', 'of', 'the', 'year', ',', 'best', 'r', '&', 'b/soul', 'or', 'rap', 'new', 'artist', ',', 'and', 'best', 'r', '&', 'b/soul', 'single', 'for', '\"', 'no', ',', 'no', ',', 'no', '\"', '.', 'the', 'group', 'released', 'their', 'multi-platinum', 'second', 'album', 'the', 'writing', \"'s\", 'on', 'the', 'wall', 'in', '1999', '.', 'the', 'record', 'features', 'some', 'of', 'the', 'group', \"'s\", 'most', 'widely', 'known', 'songs', 'such', 'as', '\"', 'bills', ',', 'bills', ',', 'bills', '\"', ',', 'the', 'group', \"'s\", 'first', 'number-one', 'single', ',', '\"', 'jumpin', \"'\", 'jumpin', \"'\", '\"', 'and', '\"', 'say', 'my', 'name', '\"', ',', 'which', 'became', 'their', 'most', 'successful', 'song', 'at', 'the', 'time', ',', 'and', 'would', 'remain', 'one', 'of', 'their', 'signature', 'songs', '.', '\"', 'say', 'my', 'name', '\"', 'won', 'the', 'best', 'r', '&', 'b', 'performance', 'by', 'a', 'duo', 'or', 'group', 'with', 'vocals', 'and', 'the', 'best', 'r', '&', 'b', 'song', 'at', 'the', '43rd', 'annual', 'grammy', 'awards', '.', 'the', 'writing', \"'s\", 'on', 'the', 'wall', 'sold', 'more', 'than', 'eight', 'million', 'copies', 'worldwide', '.', 'during', 'this', 'time', ',', 'beyonc\\xc3\\xa9', 'recorded', 'a', 'duet', 'with', 'marc', 'nelson', ',', 'an', 'original', 'member', 'of', 'boyz', 'ii', 'men', ',', 'on', 'the', 'song', '\"', 'after', 'all', 'is', 'said', 'and', 'done', '\"', 'for', 'the', 'soundtrack', 'to', 'the', '1999', 'film', ',', 'the', 'best', 'man', '.'], ['through', 'the', 'influence', 'of', 'the', 'eusebian', 'faction', 'at', 'constantinople', ',', 'an', 'arian', 'bishop', ',', 'george', 'of', 'cappadocia', ',', 'was', 'now', 'appointed', 'to', 'rule', 'the', 'see', 'of', 'alexandria', '.', 'athanasius', ',', 'after', 'remaining', 'some', 'days', 'in', 'the', 'neighbourhood', 'of', 'the', 'city', ',', 'finally', 'withdrew', 'into', 'the', 'desert', 'of', 'upper', 'egypt', ',', 'where', 'he', 'remained', 'for', 'a', 'period', 'of', 'six', 'years', ',', 'living', 'the', 'life', 'of', 'the', 'monks', ',', 'devoting', 'himself', 'to', 'the', 'composition', 'of', 'a', 'group', 'of', 'writings', ';', '\"', 'apology', 'to', 'constantius', '\"', ',', 'the', '\"', 'apology', 'for', 'his', 'flight', '\"', ',', 'the', '\"', 'letter', 'to', 'the', 'monks', '\"', ',', 'and', 'the', '\"', 'history', 'of', 'the', 'arians', '\"', '.'], ['in', 'march', '2008', ',', 'it', 'was', 'announced', 'that', 'avicenna', \"'s\", 'name', 'would', 'be', 'used', 'for', 'new', 'directories', 'of', 'education', 'institutions', 'for', 'health', 'care', 'professionals', ',', 'worldwide', '.', 'the', 'avicenna', 'directories', 'will', 'list', 'universities', 'and', 'schools', 'where', 'doctors', ',', 'public', 'health', 'practitioners', ',', 'pharmacists', 'and', 'others', ',', 'are', 'educated', '.', 'the', 'project', 'team', 'stated', '\"', 'why', 'avicenna', '?', 'avicenna', '...', 'was', '...', 'noted', 'for', 'his', 'synthesis', 'of', 'knowledge', 'from', 'both', 'east', 'and', 'west', '.', 'he', 'has', 'had', 'a', 'lasting', 'influence', 'on', 'the', 'development', 'of', 'medicine', 'and', 'health', 'sciences', '.', 'the', 'use', 'of', 'avicenna', \"'s\", 'name', 'symbolises', 'the', 'worldwide', 'partnership', 'that', 'is', 'needed', 'for', 'the', 'promotion', 'of', 'health', 'services', 'of', 'high', 'quality', '.', '\"'], ['the', 'root', 'system', 'and', 'the', 'shoot', 'system', 'are', 'interdependent', '\\xe2\\x80\\x93', 'the', 'usually', 'nonphotosynthetic', 'root', 'system', 'depends', 'on', 'the', 'shoot', 'system', 'for', 'food', ',', 'and', 'the', 'usually', 'photosynthetic', 'shoot', 'system', 'depends', 'on', 'water', 'and', 'minerals', 'from', 'the', 'root', 'system', '.', 'cells', 'in', 'each', 'system', 'are', 'capable', 'of', 'creating', 'cells', 'of', 'the', 'other', 'and', 'producing', 'adventitious', 'shoots', 'or', 'roots', '.', 'stolons', 'and', 'tubers', 'are', 'examples', 'of', 'shoots', 'that', 'can', 'grow', 'roots', '.', 'roots', 'that', 'spread', 'out', 'close', 'to', 'the', 'surface', ',', 'such', 'as', 'those', 'of', 'willows', ',', 'can', 'produce', 'shoots', 'and', 'ultimately', 'new', 'plants', '.', 'in', 'the', 'event', 'that', 'one', 'of', 'the', 'systems', 'is', 'lost', ',', 'the', 'other', 'can', 'often', 'regrow', 'it', '.', 'in', 'fact', 'it', 'is', 'possible', 'to', 'grow', 'an', 'entire', 'plant', 'from', 'a', 'single', 'leaf', ',', 'as', 'is', 'the', 'case', 'with', 'saintpaulia', ',', 'or', 'even', 'a', 'single', 'cell', '\\xe2\\x80\\x93', 'which', 'can', 'dedifferentiate', 'into', 'a', 'callus', '(', 'a', 'mass', 'of', 'unspecialised', 'cells', ')', 'that', 'can', 'grow', 'into', 'a', 'new', 'plant', '.', 'in', 'vascular', 'plants', ',', 'the', 'xylem', 'and', 'phloem', 'are', 'the', 'conductive', 'tissues', 'that', 'transport', 'resources', 'between', 'shoots', 'and', 'roots', '.', 'roots', 'are', 'often', 'adapted', 'to', 'store', 'food', 'such', 'as', 'sugars', 'or', 'starch', ',', 'as', 'in', 'sugar', 'beets', 'and', 'carrots', '.'], ['having', 'produced', 'a', 'conceptual', 'data', 'model', 'that', 'users', 'are', 'happy', 'with', ',', 'the', 'next', 'stage', 'is', 'to', 'translate', 'this', 'into', 'a', 'schema', 'that', 'implements', 'the', 'relevant', 'data', 'structures', 'within', 'the', 'database', '.', 'this', 'process', 'is', 'often', 'called', 'logical', 'database', 'design', ',', 'and', 'the', 'output', 'is', 'a', 'logical', 'data', 'model', 'expressed', 'in', 'the', 'form', 'of', 'a', 'schema', '.', 'whereas', 'the', 'conceptual', 'data', 'model', 'is', '(', 'in', 'theory', 'at', 'least', ')', 'independent', 'of', 'the', 'choice', 'of', 'database', 'technology', ',', 'the', 'logical', 'data', 'model', 'will', 'be', 'expressed', 'in', 'terms', 'of', 'a', 'particular', 'database', 'model', 'supported', 'by', 'the', 'chosen', 'dbms', '.', '(', 'the', 'terms', 'data', 'model', 'and', 'database', 'model', 'are', 'often', 'used', 'interchangeably', ',', 'but', 'in', 'this', 'article', 'we', 'use', 'data', 'model', 'for', 'the', 'design', 'of', 'a', 'specific', 'database', ',', 'and', 'database', 'model', 'for', 'the', 'modelling', 'notation', 'used', 'to', 'express', 'that', 'design', '.', ')'], ['the', 'starch', 'source', 'in', 'a', 'beer', 'provides', 'the', 'fermentable', 'material', 'and', 'is', 'a', 'key', 'determinant', 'of', 'the', 'strength', 'and', 'flavour', 'of', 'the', 'beer', '.', 'the', 'most', 'common', 'starch', 'source', 'used', 'in', 'beer', 'is', 'malted', 'grain', '.', 'grain', 'is', 'malted', 'by', 'soaking', 'it', 'in', 'water', ',', 'allowing', 'it', 'to', 'begin', 'germination', ',', 'and', 'then', 'drying', 'the', 'partially', 'germinated', 'grain', 'in', 'a', 'kiln', '.', 'malting', 'grain', 'produces', 'enzymes', 'that', 'convert', 'starches', 'in', 'the', 'grain', 'into', 'fermentable', 'sugars', '.', 'different', 'roasting', 'times', 'and', 'temperatures', 'are', 'used', 'to', 'produce', 'different', 'colours', 'of', 'malt', 'from', 'the', 'same', 'grain', '.', 'darker', 'malts', 'will', 'produce', 'darker', 'beers', '.']]\n"
     ]
    }
   ],
   "source": [
    "print(len(context_len))\n",
    "print(context_len[:10])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "86326\n"
     ]
    }
   ],
   "source": [
    "train_ques = 'data/train.question'\n",
    "\n",
    "ques_len = []\n",
    "\n",
    "with open(train_ques) as fp:  \n",
    "    lines = fp.readlines()\n",
    "    print(len(lines))\n",
    "    for line in lines:\n",
    "        words = line.split()\n",
    "        ques_len.append(words)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "86326\n",
      "[['what', 'is', 'the', 'profession', 'of', 'james', 'a.', 'beckford', '?'], ['what', 'era', 'came', 'before', 'the', 'proterozoic', 'era', '?'], ['what', 'were', 'the', 'main', 'influences', 'of', 'the', 'official', 'language', 'of', 'the', 'empire', '?'], ['who', 'started', 'a', 'revolution', 'in', 'music', 'with', 'women', '?'], ['which', 'film', 'featured', 'destiny', \"'s\", 'child', \"'s\", 'first', 'major', 'single', '?'], ['where', 'did', 'athanasius', 'remain', 'for', '6', 'years', 'while', 'living', 'as', 'a', 'monk', '?'], ['what', 'was', 'avicenna', 'known', 'for', '?'], ['what', 'are', 'roots', 'used', 'for', 'in', 'beets', 'and', 'carrots', '?'], ['in', 'what', 'form', 'is', 'the', 'output', 'of', 'a', 'logical', 'database', 'design', '?'], ['what', 'does', 'a', 'grains', 'starch', 'become', 'after', 'an', 'is', 'malted', '?']]\n"
     ]
    }
   ],
   "source": [
    "print(len(ques_len))\n",
    "print(ques_len[:10])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "151546\n"
     ]
    }
   ],
   "source": [
    "all_words = context_len[:1000]+ ques_len[:1000]\n",
    "all_words = sum(all_words, []) # flatten a list\n",
    "print(len(all_words))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "# all_words = sum(all_words, []) # flatten a list\n",
    "\n",
    "## convert to string so set can be used\n",
    "str1 = ''.join(all_words)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "# print(str1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['!', '\"', '#', '$', '%', '&', \"'\", '(', ')', '+', ',', '-', '.', '/', '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', ':', ';', '<', '=', '>', '?', '[', ']', '^', 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', 'z', '~', '\\x80', '\\x81', '\\x82', '\\x83', '\\x84', '\\x85', '\\x86', '\\x87', '\\x88', '\\x89', '\\x8a', '\\x8b', '\\x8c', '\\x8d', '\\x8e', '\\x8f', '\\x90', '\\x91', '\\x92', '\\x93', '\\x94', '\\x95', '\\x96', '\\x97', '\\x98', '\\x99', '\\x9b', '\\x9c', '\\x9d', '\\x9e', '\\x9f', '\\xa0', '\\xa1', '\\xa2', '\\xa3', '\\xa4', '\\xa5', '\\xa6', '\\xa7', '\\xa8', '\\xa9', '\\xaa', '\\xab', '\\xac', '\\xad', '\\xae', '\\xaf', '\\xb0', '\\xb1', '\\xb2', '\\xb3', '\\xb4', '\\xb5', '\\xb6', '\\xb7', '\\xb8', '\\xb9', '\\xba', '\\xbb', '\\xbc', '\\xbd', '\\xbe', '\\xbf', '\\xc2', '\\xc3', '\\xc4', '\\xc5', '\\xc6', '\\xc7', '\\xc8', '\\xc9', '\\xca', '\\xcb', '\\xcc', '\\xcd', '\\xce', '\\xcf', '\\xd0', '\\xd1', '\\xd2', '\\xd5', '\\xd6', '\\xd7', '\\xd8', '\\xd9', '\\xda', '\\xdb', '\\xe1', '\\xe2', '\\xe4', '\\xe5', '\\xe6', '\\xe7', '\\xe8', '\\xe9', '\\xef']\n"
     ]
    }
   ],
   "source": [
    "chars = sorted(set(str1))\n",
    "print(chars)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "156\n"
     ]
    }
   ],
   "source": [
    "print(len(chars))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "unique_chars = ['!', '\"', '#', '$', '%', '&', \"'\", '(', ')', '+', ',', '-', '.', '/', '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', ':', ';', '<', '=', '>', '?', '[', ']', '^', 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', 'z', '~',]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "60\n"
     ]
    }
   ],
   "source": [
    "print(len(unique_chars))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "## typical length of word\n",
    "len_words = []\n",
    "for word in all_words:\n",
    "    len_words.append(len(word))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "151546\n",
      "[11, 5, 2, 8, 1, 2, 3, 4, 5, 2]\n"
     ]
    }
   ],
   "source": [
    "print(len(len_words))\n",
    "print(len_words[:10])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "('Min:   ', 1)\n",
      "('Max:   ', 32)\n",
      "('Mean:   ', 4.6036121045755083)\n",
      "('25th percentile:   ', 2.0)\n",
      "('Median:            ', 4.0)\n",
      "('75th percentile:   ', 7.0)\n",
      "('95th percentile:   ', 10.0)\n",
      "('99th percentile:   ', 13.0)\n"
     ]
    }
   ],
   "source": [
    "len_array = np.array(len_words)\n",
    "\n",
    "print(\"Min:   \", np.min(len_array))\n",
    "print(\"Max:   \", np.max(len_array))\n",
    "print(\"Mean:   \", np.mean(len_array))\n",
    "print(\"25th percentile:   \", np.percentile(len_array, 25))\n",
    "print(\"Median:            \", np.median(len_array))\n",
    "print(\"75th percentile:   \", np.percentile(len_array, 75))\n",
    "print(\"95th percentile:   \", np.percentile(len_array, 95))\n",
    "print(\"99th percentile:   \", np.percentile(len_array, 99))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
